#!/usr/bin/python
import sys
import os
import stat
import re
import time
import datetime
import hashlib
import zipfile
import difflib
from optparse import OptionParser

LICENSE = '''
    pipe.py -- a simple pipe based text (code) search tool.
    Copyright (C) 2011, 2012, 2013, 2014 Anders Clausen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    GNU General Public License at <http://www.gnu.org/licenses/>
    for more details.
'''

HINT = """
Create a symbolic link to srch.py and fnd.py in ~/bin/.
Then search with "srch" and find files with "fnd".
"""

VERSION = "1.1.13"

IS_DOS = sys.executable.endswith(".exe")


class ShellColors:
    """Shell Color Codes"""
    def __init__(self):
        pass

    Red = '\033[00;91m'
    Green = '\033[00;92m'
    Blue = '\033[00;94m'
    Cyan = '\033[00;96m'
    White = '\033[00;97m'
    Yellow = '\033[00;93m'
    Magenta = '\033[00;95m'
    Grey = '\033[00;90m'
    Black = '\033[00;90m'
    Default = '\033[00;99m'

    RedBold = '\033[01;91m'
    GreenBold = '\033[01;92m'
    BlueBold = '\033[01;94m'
    CyanBold = '\033[01;96m'
    WhiteBold = '\033[01;97m'
    YellowBold = '\033[01;93m'
    MagentaBold = '\033[01;95m'
    GreyBold = '\033[01;90m'
    BlackBold = '\033[01;90m'
    DefaultBold = '\033[01;99m'

    EndColor = '\033[00;0m'


class PipeException(Exception):
    def __init__(self, message, error_code):
        self.message = message
        self.error_code = error_code


class CommandException(Exception):
    def __init__(self, message, command):
        self.message = message
        self.command = command


def handle_process_exception(process_function):
    def wrapper(self, **kwargs):
        try:
            process_function(self, **kwargs)
        except Exception, e:
            print >> sys.stderr, "Failed pipe stage execution", str(e), kwargs

    return wrapper


class BaseFilter:
    NONE = 0
    START_PIPE = 1
    END_PIPE = 2
    START_FILE = 3
    END_FILE = 4

    def __init__(self):
        self.next_filter = None

    def connect(self, next_filter):
        self.next_filter = next_filter

    @handle_process_exception
    def process(self, **kwargs):
        raise PipeException('Un-implemented filter process "%s"'
                            % self.__class__.__name__, 23)

    def control(self, message):
        """Push control message to next filter. Sinks must consume this."""
        self.next_filter.control(message)


class FilterSinkEndPipe(BaseFilter):
    """An empty sink filter that does nothing."""

    def __init__(self):
        BaseFilter.__init__(self)

    @handle_process_exception
    def process(self, **kwargs):
        pass

    def control(self, message):
        pass


class BaseFilterSourceStorage(BaseFilter):
    """The main storage iterator source filter base"""

    def __init__(self, start_path, top_down):
        BaseFilter.__init__(self)
        self.startPath = start_path
        self.topDown = top_down


class FilterSourceFilesAndDirs(BaseFilterSourceStorage):
    """The main file and directory iterator source filter"""

    def __init__(self, start_path, top_down):
        BaseFilterSourceStorage.__init__(self, start_path, top_down)

    @handle_process_exception
    def process(self, **kwargs):
        self.next_filter.control(BaseFilter.START_PIPE)
        for root, dirs, files in os.walk(self.startPath, topdown=self.topDown):
            for file_name in files:
                self.next_filter.process(root=root,
                                         file_name=file_name,
                                         dir_name=None,
                                         joined=os.path.join(root, file_name))
            for dir_name in dirs:
                self.next_filter.process(root=root,
                                         file_name=dir_name,
                                         dir_name=dir_name,
                                         joined=os.path.join(root, dir_name))
        self.next_filter.control(BaseFilter.END_PIPE)


class FilterSourceFiles(BaseFilterSourceStorage):
    """The main file iterator source filter"""

    def __init__(self, start_path, top_down):
        BaseFilterSourceStorage.__init__(self, start_path, top_down)

    @handle_process_exception
    def process(self, **kwargs):
        self.next_filter.control(BaseFilter.START_PIPE)
        for root, dirs, files in os.walk(self.startPath, topdown=self.topDown):
            for file_name in files:
                self.next_filter.process(root=root,
                                         file_name=file_name,
                                         dir_name=None,
                                         joined=os.path.join(root, file_name))
        self.next_filter.control(BaseFilter.END_PIPE)


class FilterSourceDirs(BaseFilterSourceStorage):
    """The main directory iterator source filter"""

    def __init__(self, start_path, top_down):
        BaseFilterSourceStorage.__init__(self, start_path, top_down)

    @handle_process_exception
    def process(self, **kwargs):
        self.next_filter.control(BaseFilter.START_PIPE)
        for root, dirs, files in os.walk(self.startPath, topdown=self.topDown):
            for dir_name in dirs:
                self.next_filter.process(root=root,
                                         file_name=dir_name,
                                         dir_name=dir_name,
                                         joined=os.path.join(root, dir_name))
        self.next_filter.control(BaseFilter.END_PIPE)


class FilterShunPath(BaseFilter):
    """A filter that removes shunned paths (root + name)"""

    def __init__(self, shun):
        BaseFilter.__init__(self)
        self.shun = re.compile(shun)

    @handle_process_exception
    def process(self, **kwargs):
        if not self.shun.search(kwargs['joined']):
            self.next_filter.process(**kwargs)


class FilterShunName(BaseFilter):
    """A filter that removes shunned files or directories."""

    def __init__(self, shun):
        BaseFilter.__init__(self)
        self.shun = re.compile(shun)

    @handle_process_exception
    def process(self, **kwargs):
        if not self.shun.search(kwargs['file_name']):
            self.next_filter.process(**kwargs)


class FilterShunRoot(BaseFilter):
    """A filter that removes shunned root paths."""

    def __init__(self, shun):
        BaseFilter.__init__(self)
        self.shun = re.compile(shun)

    @handle_process_exception
    def process(self, **kwargs):
        if not self.shun.search(kwargs['root']):
            self.next_filter.process(**kwargs)


class FilterFileType(BaseFilter):
    """The file type filter. Selects only the files in fileTypes."""

    def __init__(self, file_types):
        BaseFilter.__init__(self)
        self.fileTypes = [".%s" % s.strip() for s in file_types.split(",")]

    @handle_process_exception
    def process(self, **kwargs):
        name, ext = os.path.splitext(kwargs['file_name'])
        if ext.lower() in self.fileTypes:
            self.next_filter.process(**kwargs)


class FilterFileSize(BaseFilter):
    """Get the file size and push it on the pipe data."""

    def __init__(self):
        BaseFilter.__init__(self)
        self.size = 0

    @handle_process_exception
    def process(self, **kwargs):
        """Push file size on args and call next filter."""
        kwargs['file_size'] = os.path.getsize(kwargs['joined'])
        self.next_filter.process(**kwargs)


class FilterFileSizeAbove(BaseFilter):
    """Filter that gets the file size and only pushes if above size."""

    def __init__(self, size):
        BaseFilter.__init__(self)
        self.size = size

    @handle_process_exception
    def process(self, **kwargs):
        if kwargs['file_size'] >= self.size:
            self.next_filter.process(**kwargs)


class FilterFileSizeBelow(BaseFilter):
    """Filter that gets the file size and only pushes if below size."""

    def __init__(self, size):
        BaseFilter.__init__(self)
        self.size = size

    @handle_process_exception
    def process(self, **kwargs):
        if kwargs['file_size'] <= self.size:
            self.next_filter.process(**kwargs)


class FilterGenerateMd5(BaseFilter):
    """A filter that generates md5 sums and pushes into args as 'file_hash'."""

    def __init__(self):
        BaseFilter.__init__(self)

    @handle_process_exception
    def process(self, **kwargs):
        kwargs['file_hash'] = generate_md5(kwargs['joined'])
        self.next_filter.process(**kwargs)


class FilterGenerateSha1(BaseFilter):
    """A filter that generates sha1 sums and pushes into args as
    'file_hash'."""

    def __init__(self):
        BaseFilter.__init__(self)

    @handle_process_exception
    def process(self, **kwargs):
        kwargs['file_hash'] = generate_sha1(kwargs['joined'])
        self.next_filter.process(**kwargs)


class FilterFileReadLines(BaseFilter):
    """The filter that reads the lines of a file and pushes them as 'line'."""

    def __init__(self):
        BaseFilter.__init__(self)

    @handle_process_exception
    def process(self, **kwargs):
        if kwargs['dir_name']:
            return

        self.next_filter.control(BaseFilter.START_FILE)
        read_file = open(kwargs['joined'])
        for kwargs["index"], kwargs["line"] in \
                enumerate(read_file.readlines()):
            self.next_filter.process(**kwargs)
        read_file.close()
        self.next_filter.control(BaseFilter.END_FILE)


class BaseFilterMatchSingle(BaseFilter):
    """A base class match filter for single lines."""

    def __init__(self, regex, ignore_case):
        BaseFilter.__init__(self)
        self.regex = []
        rx = '|'.join(regex)

        try:
            self.regex.append(re.compile(rx, flags=re.IGNORECASE)
                              if ignore_case else re.compile(rx))
        except Exception, e:
            raise PipeException('Failed to compile regex, "%s"' % str(e), 2)


class BaseFilterMatchBlock(BaseFilterMatchSingle):
    """A base class match filter for blocks, i.e. start and end matches."""

    def __init__(self, regex, regex_end, ignore_case):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)
        try:
            self.regexEnd = re.compile(regex_end, flags=re.IGNORECASE) \
                if ignore_case else re.compile(regex_end)
        except Exception, e:
            raise PipeException('Failed to compile regex, "%s"' % str(e), 3)

        self.inBlock = False
        self.newFile = False

    def control(self, message):
        if message == BaseFilter.END_FILE:
            self.inBlock = False
        elif message == BaseFilter.START_FILE:
            self.newFile = True

        self.next_filter.control(message)


class FilterRemoveLine(BaseFilterMatchSingle):
    """A remove line filter. Removes matching lines."""

    def __init__(self, regex, ignore_case):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)

    @handle_process_exception
    def process(self, **kwargs):
        if not self.regex[0].search(kwargs['line']):
            self.next_filter.process(**kwargs)


class FilterSelectLine(BaseFilterMatchSingle):
    """A select line filter. Selects matching lines."""

    def __init__(self, regex, ignore_case):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)

    @handle_process_exception
    def process(self, **kwargs):
        span = [m.span() for m in self.regex[0].finditer(kwargs['line'])]
        if span:
            span.reverse()
            kwargs['span'] = span
            self.next_filter.process(**kwargs)


class FilterSelectLines(BaseFilterMatchSingle):
    """A select line filter. Selects matching lines."""

    def __init__(self, regex, ignore_case, before, after):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)
        self.before = before if before else 0
        self.after = after if after else 0
        self.lines = []
        self.to_process = []
        self.index = 0

    def _get_the_before(self, i):
        the_before = []
        before_index = max(i - self.before, 0)
        for j in xrange(i - before_index):
            the_before.append(self.lines[before_index + j])
        return the_before

    def _get_the_after(self, i):
        the_after = []
        after_count = min(self.after, len(self.lines) - i - 1)
        for j in xrange(after_count):
            the_after.append(self.lines[i + j + 1])
        return the_after

    def _get_the_kwargs(self, i):
        the_before = self._get_the_before(i)
        the_after = self._get_the_after(i)
        the_kwargs = self.lines[i]
        the_kwargs['before'] = the_before
        the_kwargs['after'] = the_after
        return the_kwargs

    @handle_process_exception
    def process(self, **kwargs):
        span = [m.span() for m in self.regex[0].finditer(kwargs['line'])]
        if span:
            span.reverse()
            kwargs['span'] = span
            self.to_process.append(self.index)

        self.lines.append(kwargs)

        if self.to_process and self.index - self.after == self.to_process[0]:
            the_kwargs = self._get_the_kwargs(self.to_process.pop(0))
            self.next_filter.process(**the_kwargs)

        self.index += 1

    def control(self, message):
        if message == BaseFilter.END_FILE:
            for i in self.to_process:
                the_kwargs = self._get_the_kwargs(i)
                self.next_filter.process(**the_kwargs)
            self.lines = []
            self.to_process = []
            self.index = 0

        self.next_filter.control(message)


class FilterSelectFileName(BaseFilterMatchSingle):
    """Select matching file names."""

    def __init__(self, regex, ignore_case):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)

    @handle_process_exception
    def process(self, **kwargs):
        if self.regex[0].search(kwargs['file_name']):
            self.next_filter.process(**kwargs)


class FilterReplaceInLineAll(BaseFilterMatchSingle):
    """A filter for replacing all matches within a line."""

    def __init__(self, regex, replacee, replacer, ignore_case):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)
        self.replacer = replacer
        try:
            self.replacee = re.compile(replacee, flags=re.IGNORECASE) \
                if ignore_case else re.compile(replacee)
        except Exception, e:
            raise PipeException('Failed to compile replacee, "%s"' %
                                str(e), 12)

    @handle_process_exception
    def process(self, **kwargs):
        if self.regex[0].search(kwargs['line']):
            kwargs['line'] = re.sub(self.replacee,
                                    self.replacer,
                                    kwargs['line'])
        self.next_filter.process(**kwargs)


class FilterSelectFirstLine(BaseFilterMatchSingle):
    """A filter that selects only the first line of a file."""

    def __init__(self, regex, ignore_case):
        BaseFilterMatchSingle.__init__(self, regex, ignore_case)
        self.unmatched = True

    @handle_process_exception
    def process(self, **kwargs):
        if self.unmatched:
            if self.regex[0].search(kwargs['line']):
                self.next_filter.process(**kwargs)
                self.unmatched = False

    def control(self, message):
        if message == BaseFilter.START_FILE:
            self.unmatched = True

        self.next_filter.control(message)


class FilterRemoveBlock(BaseFilterMatchBlock):
    """A filter to remove all lines between first and last match, inclusive."""

    def __init__(self, regex, regex_end, ignore_case):
        BaseFilterMatchBlock.__init__(self, regex, regex_end, ignore_case)

    @handle_process_exception
    def process(self, **kwargs):
        line = kwargs['line']

        if self.inBlock:
            if self.regexEnd.search(line):
                self.inBlock = False
        else:
            if self.regex[0].search(line):
                self.inBlock = True
            else:
                self.next_filter.process(**kwargs)


class FilterSelectBlock(BaseFilterMatchBlock):
    """A filter to select all lines between first and last match, inclusive."""

    def __init__(self, regex, regex_end, ignore_case):
        BaseFilterMatchBlock.__init__(self, regex, regex_end, ignore_case)

    @handle_process_exception
    def process(self, **kwargs):
        line = kwargs['line']

        if self.inBlock:
            self.next_filter.process(**kwargs)
            if self.regexEnd.search(line):
                self.inBlock = False
        else:
            if self.regex[0].search(line):
                self.next_filter.process(**kwargs)
                self.inBlock = True


class FilterSortLinesBuffered(BaseFilter):
    """Sort lines or file names. Buffered, so output waits until end."""

    def __init__(self, reverse):
        BaseFilter.__init__(self)
        self.reverse = reverse
        self.buffer = []

    @handle_process_exception
    def process(self, **kwargs):
        self.buffer.append(kwargs)

    def control(self, message):
        if message == BaseFilter.END_PIPE:
            arr = sorted(self.buffer,
                         key=lambda arg: arg['line'].strip(),
                         reverse=self.reverse)

            for kwargs in arr:
                self.next_filter.process(**kwargs)

        # This means that content and control messages arrive out of order.
        # Control messages are sent before any content is pushed into the pipe,
        # Except for the END_PIPE message.
        self.next_filter.control(message)


class FilterSortFilesBuffered(BaseFilter):
    """Sort lines or file names. Buffered, so output waits until end."""

    def __init__(self, reverse):
        BaseFilter.__init__(self)
        self.reverse = reverse
        self.buffer = []

    @handle_process_exception
    def process(self, **kwargs):
        self.buffer.append(kwargs)

    def control(self, message):
        if message == BaseFilter.END_PIPE:
            arr = sorted(self.buffer,
                         key=lambda arg: os.path.getsize(arg['joined']),
                         reverse=self.reverse)

            for kwargs in arr:
                self.next_filter.process(**kwargs)

        # This means that content and control messages arrive out of order.
        # Control messages are sent before any content is pushed into the pipe,
        # Except for the END_PIPE message.
        self.next_filter.control(message)


class FilterPrintLinePlain(BaseFilter):
    """A filter for printing one plain line."""

    def __init__(self):
        BaseFilter.__init__(self)

    @staticmethod
    def color_format_string(kwargs, span, color_override=None):
        l = kwargs['line']
        color = color_override if color_override else ShellColors.RedBold
        for sp in span:
            l = l[:sp[1]] + ShellColors.EndColor + l[sp[1]:]
            l = l[:sp[0]] + color + l[sp[0]:]
        return l

    @staticmethod
    def _do_print(kwargs, color_override=None):
        if options.colors:
            print '%s%s%s:%s' % (ShellColors.Yellow, kwargs['joined'],
                                 ShellColors.GreyBold, ShellColors.EndColor),

            span = kwargs.get('span', None)
            if not span:
                print '%s' % kwargs['line'],
            else:
                print '%s' %\
                      FilterPrintLinePlain.color_format_string(kwargs,
                                                               span,
                                                               color_override),
        else:
            print '%s: %s' % (kwargs['joined'],
                              kwargs['line']),

    @handle_process_exception
    def process(self, **kwargs):
        before = kwargs.get('before', [])
        after = kwargs.get('after', [])
        if before or after:
            for b in before:
                FilterPrintLinePlain.\
                    _do_print(b, color_override=ShellColors.GreenBold)
            FilterPrintLinePlain._do_print(kwargs)
            for a in after:
                FilterPrintLinePlain.\
                    _do_print(a, color_override=ShellColors.GreenBold)
            print ""
        else:
            FilterPrintLinePlain._do_print(kwargs)
        self.next_filter.process(**kwargs)


class FilterPrintLinePlainWithNumbers(BaseFilter):
    """A filter for printing one plain line with numbers."""

    def __init__(self):
        BaseFilter.__init__(self)

    @staticmethod
    def _do_print(kwargs, color_override=None):
        if options.colors:
            print '%s%s%s:%s%d%s:%s' % (ShellColors.Yellow, kwargs['joined'],
                                        ShellColors.GreyBold,
                                        ShellColors.BlueBold,
                                        kwargs['index'] + 1,
                                        ShellColors.GreyBold,
                                        ShellColors.EndColor),

            span = kwargs.get('span', None)
            if not span:
                print '%s' % kwargs['line'],
            else:
                print '%s' % \
                      FilterPrintLinePlain.color_format_string(kwargs,
                                                               span,
                                                               color_override),
        else:
            print '%s:%d: %s' % (kwargs['joined'], kwargs['index'] + 1,
                                 kwargs['line']),

    @handle_process_exception
    def process(self, **kwargs):
        before = kwargs.get('before', [])
        after = kwargs.get('after', [])
        if before or after:
            for b in before:
                FilterPrintLinePlainWithNumbers.\
                    _do_print(b, color_override=ShellColors.GreenBold)
            FilterPrintLinePlainWithNumbers._do_print(kwargs)
            for a in after:
                FilterPrintLinePlainWithNumbers.\
                    _do_print(a, color_override=ShellColors.GreenBold)
            print ""
        else:
            FilterPrintLinePlainWithNumbers._do_print(kwargs)

        self.next_filter.process(**kwargs)


class FilterPrintFilePlain(BaseFilter):
    """A filter for printing plain file names."""

    def __init__(self):
        BaseFilter.__init__(self)

    @handle_process_exception
    def process(self, **kwargs):
        print '%s' % (kwargs['joined'])
        self.next_filter.process(**kwargs)


class BaseFilterPrintCute(BaseFilter):
    """A base filter for cute printing."""

    def __init__(self):
        BaseFilter.__init__(self)
        self.first_visit = None

    def control(self, message):
        if message == BaseFilter.START_FILE:
            self.first_visit = True
        self.next_filter.control(message)


class FilterPrintFileCute(BaseFilterPrintCute):
    """A filter for printing cute file names."""

    def __init__(self):
        BaseFilterPrintCute.__init__(self)

    @staticmethod
    def get_colors():
        size_color = ''
        date_color = ''
        file_color = ''
        hash_color = ''
        end_color = ''
        if options.colors:
            size_color = ShellColors.Green
            date_color = ShellColors.Blue
            file_color = ShellColors.YellowBold
            hash_color = ShellColors.Red
            end_color = ShellColors.EndColor

        return size_color, date_color, file_color, hash_color, end_color

    @handle_process_exception
    def process(self, **kwargs):
        root = kwargs['root']
        file_name = kwargs['file_name']
        dir_name = kwargs['dir_name']

        # dir_name is set only if dir, otherwise None.
        if dir_name:
            print "Directory:", dir_name, "path", root
        else:
            size_color, date_color, file_color, hash_color, end_color =\
                FilterPrintFileCute.get_colors()
            joined = kwargs['joined']
            size = os.path.getsize(joined)
            access_time = time.ctime(os.path.getatime(joined))
            print '%s% 8s %s%s %s%s%s %s%s' % (size_color,
                                               convert_from_bytes(size),
                                               date_color,
                                               access_time,
                                               file_color,
                                               file_name,
                                               hash_color,
                                               kwargs.get('file_hash', ""),
                                               end_color)
        self.next_filter.process(**kwargs)


class FilterPrintFileDetail(BaseFilterPrintCute):
    """A filter for printing detailed file names."""

    def __init__(self):
        BaseFilterPrintCute.__init__(self)

    @handle_process_exception
    def process(self, **kwargs):
        root = kwargs['root']
        dir_name = kwargs['dir_name']

        # dir_name is set only if dir, otherwise None.
        if dir_name:
            print "Directory:", dir_name, "path", root
        else:
            size_color, date_color, file_color, hash_color, end_color =\
                FilterPrintFileCute.get_colors()
            joined = kwargs['joined']
            size = os.path.getsize(joined)
            print '%s%012d %s%s%s %s%s' % (size_color,
                                           size,
                                           file_color,
                                           os.path.abspath(joined),
                                           hash_color,
                                           kwargs.get('file_hash', ""),
                                           end_color)
        self.next_filter.process(**kwargs)


class FilterPrintLineCute(BaseFilterPrintCute):
    """A filter for printing cute lines."""

    def __init__(self):
        BaseFilterPrintCute.__init__(self)

    @staticmethod
    def print_cute_file_name(kwargs):
        size_color, date_color, file_color, hash_color, end_color = \
            FilterPrintFileCute.get_colors()
        print ""
        print "%s%s%s: %s%s%s" % (file_color,
                                  kwargs['joined'],
                                  date_color,
                                  hash_color,
                                  kwargs.get('file_hash', ""),
                                  end_color)

    @staticmethod
    def _do_print(kwargs, color_override=None):
        span = kwargs.get('span', None)
        if span and options.colors:
            l = FilterPrintLinePlain.color_format_string(kwargs, span,
                                                         color_override)
        else:
            l = kwargs['line']
        print "    %s" % l,

    @handle_process_exception
    def process(self, **kwargs):
        if self.first_visit:
            FilterPrintLineCute.print_cute_file_name(kwargs)

        before = kwargs.get('before', [])
        after = kwargs.get('after', [])
        if before or after:
            for b in before:
                FilterPrintLineCute.\
                    _do_print(b, color_override=ShellColors.GreenBold)
            FilterPrintLineCute._do_print(kwargs)
            for a in after:
                FilterPrintLineCute.\
                    _do_print(a, color_override=ShellColors.GreenBold)
            print ""
        else:
            FilterPrintLineCute._do_print(kwargs)

        self.first_visit = False
        self.next_filter.process(**kwargs)


class FilterPrintLineCuteWithNumbers(BaseFilterPrintCute):
    """A filter for printing cute lines with numbers."""

    def __init__(self):
        BaseFilterPrintCute.__init__(self)

    @staticmethod
    def _do_print(kwargs, color_override=None):
        span = kwargs.get('span', None)
        if span and options.colors:
            l = FilterPrintLinePlain.color_format_string(kwargs, span,
                                                         color_override)
            number_color = ShellColors.Blue
            end_color = ShellColors.EndColor
            colon_color = ShellColors.Grey
        else:
            l = kwargs['line']
            number_color = ''
            end_color = ''
            colon_color = ''
        print "%s% 4d%s:%s %s" % (number_color,
                                  kwargs['index'] + 1,
                                  colon_color,
                                  end_color,
                                  l),

    @handle_process_exception
    def process(self, **kwargs):
        if self.first_visit:
            FilterPrintLineCute.print_cute_file_name(kwargs)

        before = kwargs.get('before', [])
        after = kwargs.get('after', [])
        if before or after:
            for b in before:
                FilterPrintLineCuteWithNumbers.\
                    _do_print(b, color_override=ShellColors.GreenBold)
            FilterPrintLineCuteWithNumbers._do_print(kwargs)
            for a in after:
                FilterPrintLineCuteWithNumbers.\
                    _do_print(a, color_override=ShellColors.GreenBold)
            print ""
        else:
            FilterPrintLineCuteWithNumbers._do_print(kwargs)

        self.first_visit = False
        self.next_filter.process(**kwargs)


class BaseFilterWrite(BaseFilter):
    """Base for file write filters."""

    def __init__(self, remove_trailing, line_endings):
        BaseFilter.__init__(self)
        self.file_name = None
        self.lines = None
        self.remove_trailing = remove_trailing
        self.line_ending = "\r\n" if line_endings else "\n"

    @handle_process_exception
    def process(self, **kwargs):
        if not self.file_name:
            self.file_name = kwargs['joined']
            self.lines = []

        self.lines.append(kwargs['line'])
        self.next_filter.process(**kwargs)

    def control(self, message):
        if message == BaseFilter.END_FILE:
            self.write_file()
            self.file_name = None

        self.next_filter.control(message)

    def write_file(self):
        raise PipeException('Un-implemented write file "%s"'
                            % self.__class__.__name__, 4)


class FilterWriteFileInplace(BaseFilterWrite):
    """Filter for writing lines to a file inplace, i.e. overwrite."""

    def __init__(self, remove_trailing, line_endings):
        BaseFilterWrite.__init__(self, remove_trailing, line_endings)

    def write_file(self):
        if not self.file_name:
            return

        file_attributes = os.stat(self.file_name)[0]
        if not (file_attributes & stat.S_IWRITE):
            sys.stderr.write('File "%s" is not writable. Skipping.\n' %
                             self.file_name)
            return

        out_file = open(self.file_name, "w")
        if self.remove_trailing:
            out_file.writelines(['%s%s' % (l.rstrip(), self.line_ending)
                                for l in self.lines])
        else:
            out_file.writelines(self.lines)
        out_file.close()


class FilterWriteFileNew(BaseFilterWrite):
    """Filter for writing lines to a new (created) file name."""

    def __init__(self, remove_trailing, line_endings):
        BaseFilterWrite.__init__(self, remove_trailing, line_endings)

    def write_file(self):
        name, ext = os.path.splitext(self.file_name)
        new_name = None
        for i in xrange(1, 101):
            new_name = "%s_out%d%s" % (name, i, ext)
            if not os.path.exists(new_name):
                break
            new_name = None
        if not new_name:
            raise PipeException('Could not generate a unique file name for '
                                '"%s"' % self.file_name, 6)

        out_file = open(new_name, "w")
        if self.remove_trailing:
            out_file.writelines(['%s%s' % (l.rstrip(), self.line_ending)
                                for l in self.lines])
        else:
            out_file.writelines(self.lines)
        out_file.close()


class FilterExecute(BaseFilter):
    """Filter for executing python code provided as an argument."""

    def __init__(self, execute_line):
        BaseFilter.__init__(self)
        try:
            self.execute_line = compile(execute_line, '', 'exec')
        except Exception, e:
            raise PipeException('Failed to compile execute line, "%s"'
                                % str(e), 5)
        self.kwargs = None

    def v(self, name):
        """Return the value of the provided variable name, or None."""
        return self.kwargs.get(name, None)

    @handle_process_exception
    def process(self, **kwargs):
        self.kwargs = kwargs

        root = kwargs.get('root', None)
        file_name = kwargs.get('file_name', None)
        dir_name = kwargs.get('dir_name', None)
        file_hash = kwargs.get('file_hash', None)
        index = kwargs.get('index', None)
        line = kwargs.get('line', None)
        joined = kwargs.get('joined', None)
        file_size = kwargs.get('file_size', None)
        exec self.execute_line
        self.next_filter.process(**kwargs)


class FilterZip(BaseFilter):
    """Filter to write matching files in to a named zip file."""

    def __init__(self, zip_name, the_time):
        BaseFilter.__init__(self)
        if zip_name == "--time":
            zip_name = "archive.zip"
            the_time = True

        zip_root, ext = os.path.splitext(zip_name)
        if ext == "":
            ext = ".zip"

        if the_time:
            self.zip = "%s_%s%s" %\
                       (zip_root,
                        datetime.datetime.now().isoformat().replace(":", "_")
                        .replace(".", "-"), ext)
        else:
            self.zip = "%s%s" % (zip_root, ext)

        if os.path.exists(self.zip):
            raise PipeException('Zip file "%s" alredy exists' % self.zip, 22)
        self.args = {}

    @handle_process_exception
    def process(self, **kwargs):
        if not kwargs['dir_name']:
            self.args[kwargs['joined']] = kwargs
        self.next_filter.process(**kwargs)

    def control(self, message):
        if message == BaseFilter.END_PIPE:
            zf = zipfile.ZipFile(self.zip, "w")
            for k in self.args.iterkeys():
                print "Adding", k
                zf.write(k, k, zipfile.ZIP_DEFLATED)
            zf.close()
            print "Wrote %s, size: %s" % (self.zip,
                                          convert_from_bytes(
                                              os.path.getsize(self.zip)))
        self.next_filter.control(message)


class FilterWordCount(BaseFilter):
    """Filter for counting lines, words, and letters of match."""

    def __init__(self):
        BaseFilter.__init__(self)
        self.letters = 0
        self.words = 0
        self.lines = 0

    @handle_process_exception
    def process(self, **kwargs):
        line = kwargs['line']
        self.lines += 1
        self.words += len(line.split())
        self.letters += len(line)
        self.next_filter.process(**kwargs)

    def control(self, message):
        if message == BaseFilter.END_PIPE:
            print "% 7d% 8d% 8d" % (self.lines, self.words, self.letters)

        self.next_filter.control(message)


class FilterWordCountFiles(BaseFilter):
    """Filter for counting lines, words, and letters in files."""

    def __init__(self, sort, reverse):
        BaseFilter.__init__(self)
        self.sort = sort
        self.reverse = reverse
        self.args = []

    @handle_process_exception
    def process(self, **kwargs):
        if not kwargs['dir_name']:
            self.args.append(kwargs)
        self.next_filter.process(**kwargs)

    def control(self, message):
        if message == BaseFilter.END_PIPE:
            for kwargs in self.args:
                lines = 0
                words = 0
                letters = 0
                f = open(kwargs['joined'])
                for line in f.readlines():
                    lines += 1
                    words += len(line.split())
                    letters += len(line)
                kwargs['lineCount'] = lines
                kwargs['wordCount'] = words
                kwargs['letterCount'] = letters

        if self.sort:
            self.args = sorted(self.args,
                               key=lambda the_arg: the_arg['lineCount'],
                               reverse=self.reverse)

        for arg in self.args:
            print "% 7d% 8d% 8d %s" % (arg['lineCount'],
                                       arg['wordCount'],
                                       arg['letterCount'],
                                       arg['joined'])
        self.next_filter.control(message)


class FilterFindDuplicates(BaseFilter):
    """Filter for finding duplicate files using file hash."""

    def __init__(self):
        BaseFilter.__init__(self)
        self.args = {}

    @handle_process_exception
    def process(self, **kwargs):
        the_hash = kwargs['file_hash']
        kwargs_list = self.args.get(the_hash, [])
        kwargs_list.append(kwargs)
        self.args[the_hash] = kwargs_list
        self.next_filter.process(**kwargs)

    def control(self, message):
        if message == BaseFilter.END_PIPE:
            unsorted_dups = [args for args in self.args.itervalues()
                             if len(args) > 1]
            sorted_dups = sorted(unsorted_dups,
                                 key=lambda arg:
                                 len(arg) * os.path.getsize(arg[0]['joined']),
                                 reverse=True)
            for dups in sorted_dups:
                joined = dups[0]['joined']
                size = os.path.getsize(joined)
                one_size = convert_from_bytes(size)
                total_size = convert_from_bytes(size * len(dups))
                print ""
                print "Equal with hash:", dups[0]['file_hash']
                print "total size: %s," % total_size, "file size: %s," %\
                                                      one_size,
                print "count: %s" % len(dups)
                for dup in dups:
                    print dup['joined']

            if not unsorted_dups:
                print ""
                print "Did not find any duplicate files"

        self.next_filter.control(message)


def convert_to_bytes(byte_string):
    try:
        if byte_string.isdigit():
            return int(byte_string)
        if byte_string.endswith('kB'):
            return int(byte_string[:len(byte_string) - 2]) * 1000
        if byte_string.endswith('K'):
            return int(byte_string[:len(byte_string) - 1]) * 1024
        if byte_string.endswith('MB'):
            return int(byte_string[:len(byte_string) - 2]) * 1000000
        if byte_string.endswith('M'):
            return int(byte_string[:len(byte_string) - 1]) * 1048576
        if byte_string.endswith('GB'):
            return int(byte_string[:len(byte_string) - 2]) * 1000000000
        if byte_string.endswith('G'):
            return int(byte_string[:len(byte_string) - 1]) * 1073741824
        if byte_string.endswith('TB'):
            return int(byte_string[:len(byte_string) - 2]) * 1000000000000
        if byte_string.endswith('T'):
            return int(byte_string[:len(byte_string) - 1]) * 1099511627776
    except Exception, e:
        raise PipeException("Failed to convert %s. (%s)" %
                            (byte_string, str(e)), 17)

    raise PipeException("Failed to convert %s." % byte_string, 18)


def convert_from_bytes(the_bytes):
    if the_bytes < 1024:
        return "%d" % the_bytes
    if the_bytes < 1048576:
        return "%dK" % (the_bytes / 1024)
    if the_bytes < 1073741824:
        return "%dM" % (the_bytes / 1048576)
    if the_bytes < 1099511627776:
        return "%dG" % (the_bytes / 1073741824)
    else:
        return "%dT" % (the_bytes / 1099511627776)


def generate_hash(the_file, m):
    if os.path.isdir(the_file):
        return "%s is a directory (in generateHash)" % the_file

    f = None
    try:
        f = open(the_file, "rb")
        data = f.read(1024)
        while data:
            m.update(data)
            data = f.read(1024)
    except IOError, e:
        return "%s gave IOError (in generateHash), %s" % (the_file, e.message)
    except Exception, e:
        raise PipeException("Failed to generate hash for %s (%s)" %
                            (the_file, str(e)), 19)
    finally:
        if f:
            f.close()

    return m.hexdigest()


def generate_md5(the_file):
    return generate_hash(the_file, hashlib.md5())


def generate_sha1(the_file):
    return generate_hash(the_file, hashlib.sha1())


def build_pipe_from_string(build):
    import inspect

    filters = []
    filter_dict = {}
    filter_dict_short = {}
    for key, val in globals().iteritems():
        if key.startswith("Filter"):
            for k, v in inspect.getmembers(val):
                if k == "__init__":
                    arguments, varargs, keywords, defaults =\
                        inspect.getargspec(v)
                    filter_dict[key] = arguments[1:]
                    filter_dict_short[key[6:].lower()] = key

    index = 0
    while index < len(build):
        the_filter = build[index]
        filter_name = the_filter if the_filter in filter_dict.keys() \
            else filter_dict_short.get(the_filter.lower(), "")
        index += 1

        arguments = filter_dict.get(filter_name, None)
        if arguments is None:
            raise PipeException('No such filter: "%s"' % the_filter, 25)

        eval_string = "%s(%s)" %\
                      (filter_name, ", ".join(["%s=%s" %
                                               (arg, build[index + i])
                                               for i, arg in
                                               enumerate(arguments)]))
        index += len(arguments)
        filter_object = eval(eval_string)
        filters.append(filter_object)

    return filters


def get_filters():
    if options.build:
        return build_pipe_from_string(options.build)

    filters = []

    # The source filter
    if options.dirsonly:
        filters.append(FilterSourceDirs(start_path=options.path,
                                        top_down=not options.bottomup))
    elif options.dirs:
        filters.append(FilterSourceFilesAndDirs(start_path=options.path,
                                                top_down=not options.bottomup))
    else:
        filters.append(FilterSourceFiles(start_path=options.path,
                                         top_down=not options.bottomup))

    for shun in options.shunpaths:
        filters.append(FilterShunPath(shun=shun))

    for shun in options.shunnames:
        filters.append(FilterShunName(shun=shun))

    for shun in options.shunroots:
        filters.append(FilterShunRoot(shun=shun))

    # The file type selector filter
    if not options.all:
        filters.append(FilterFileType(file_types=options.types))

    if options.size:
        filters.append(FilterFileSize())
        if options.below:
            filters.append(FilterFileSizeBelow(
                size=convert_to_bytes(options.size)))
        else:
            filters.append(FilterFileSizeAbove(
                size=convert_to_bytes(options.size)))

    # Match file names
    if options.fnd or options.dups:
        if options.regex != [""]:
            filters.append(FilterSelectFileName(
                regex=options.regex,
                ignore_case=options.ignorecase))

        if options.dups or options.sha1 or options.md5:
            filters.append(FilterGenerateSha1())

        if options.sort:
            filters.append(FilterSortFilesBuffered(reverse=options.reverse))

        if options.zip:
            filters.append(FilterZip(zip_name=options.zip,
                                     the_time=options.time))
        elif options.dups:
            filters.append(FilterFindDuplicates())
        elif options.wc:
            filters.append(FilterWordCountFiles(sort=options.sort,
                                                reverse=options.reverse))
        else:
            if options.cute:
                filters.append(FilterPrintFileCute())
            elif options.detail:
                filters.append(FilterPrintFileDetail())
            else:
                filters.append(FilterPrintFilePlain())

        filters.append(FilterSinkEndPipe())
        return filters

    if options.md5:
        filters.append(FilterGenerateMd5())
    elif options.sha1:
        filters.append(FilterGenerateSha1())

    # The read line iterator filter
    filters.append(FilterFileReadLines())

    # Middle filter. Can select or remove single or block.
    if options.files:
        filters.append(FilterSelectFirstLine(regex=options.regex,
                                             ignore_case=options.ignorecase))
    elif options.end:
        if options.remove:
            filters.append(FilterRemoveBlock(regex=options.regex,
                                             regex_end=options.end,
                                             ignore_case=options.ignorecase))
        else:
            filters.append(FilterSelectBlock(regex=options.regex,
                                             regex_end=options.end,
                                             ignore_case=options.ignorecase))
    elif options.replacer:
        filters.append(FilterReplaceInLineAll(regex=options.regex,
                                              replacee=options.replacee if
                                              options.replacee else
                                              options.regex[0],
                                              replacer=options.replacer,
                                              ignore_case=options.ignorecase))
    else:
        if options.remove:
            filters.append(FilterRemoveLine(regex=options.regex,
                                            ignore_case=options.ignorecase))
        elif options.before or options.after:
            filters.append(FilterSelectLines(regex=options.regex,
                                             ignore_case=options.ignorecase,
                                             before=options.before,
                                             after=options.after))
        else:
            filters.append(FilterSelectLine(regex=options.regex,
                                            ignore_case=options.ignorecase))

    if options.sort:
        if options.files:
            filters.append(FilterSortFilesBuffered(reverse=options.reverse))
        else:
            filters.append(FilterSortLinesBuffered(reverse=options.reverse))

    # The sink filter. Print to console or to file.
    if options.zip:
        filters.append(FilterZip(zip_name=options.zip, the_time=options.time))
    elif options.exe:
        filters.append(FilterExecute(execute_line=options.exe))
    elif options.newfile:
        filters.append(FilterWriteFileNew(remove_trailing=options.rstrip,
                                          line_endings=options.crlf))
    elif options.inplace:
        filters.append(FilterWriteFileInplace(remove_trailing=options.rstrip,
                                              line_endings=options.crlf))
    elif options.wc:
        filters.append(FilterWordCount())
    else:
        if options.cute or options.detail:
            if options.numbers:
                filters.append(FilterPrintLineCuteWithNumbers())
            elif options.files:
                if options.detail:
                    filters.append(FilterPrintFileDetail())
                else:
                    filters.append(FilterPrintFileCute())
            else:
                filters.append(FilterPrintLineCute())

        else:
            if options.files:
                filters.append(FilterPrintFilePlain())
            elif options.numbers:
                filters.append(FilterPrintLinePlainWithNumbers())
            else:
                filters.append(FilterPrintLinePlain())

    filters.append(FilterSinkEndPipe())
    return filters


def build_pipe():
    filters = get_filters()

    if options.pipeinfo:
        pipe_info(filters)
        return FilterSinkEndPipe()

    # Connect
    for index in xrange(len(filters) - 1):
        filters[index].connect(filters[index + 1])

    return filters[0]


def filter_info():
    import inspect

    for key, val in globals().iteritems():
        if key.startswith("Filter"):
            print ' - %s -' % key
            print '"%s"' % val.__doc__
            for k, v in inspect.getmembers(val):
                if k == "__init__":
                    args, varargs, keywords, defaults = inspect.getargspec(v)
                    print "Init: (%s)" % \
                          (", ".join(args[1:]) if len(args) > 1 else "")
            print ""


def pipe_info(the_pipe):
    import inspect

    print ""
    for i, obj in enumerate(the_pipe):
        print ' - %s -' % obj.__class__.__name__
        print '"%s"' % obj.__doc__
        args, varargs, keywords, defaults = inspect.getargspec(obj.__init__)
        print "Init: (%s)" % (", ".join(args[1:]) if len(args) > 1 else "")
        print ""
        if i < len(the_pipe) - 1:
            print '          |'
            print '          V'
            print ""


def diff(args):
    if len(args) != 2:
        raise CommandException("Wrong number of files to diff (" +
                               str(len(args)) + ")", "diff")

    for f in args:
        if not os.path.isfile(f):
            raise CommandException("Could not find file '" + f + "'", "diff")

    try:
        context_lines = int(options.difflines)
    except ValueError:
        raise CommandException("diff context lines was not parsable '" +
                               options.difflines + "'", "diff")

    # Adapted from the diff.py example
    from_time = time.ctime(os.stat(args[0]).st_mtime)
    to_time = time.ctime(os.stat(args[1]).st_mtime)
    from_lines = open(args[0], 'U').readlines()
    to_lines = open(args[1], 'U').readlines()

    diff_type = options.difftype.lower()
    if diff_type in ('u', 'unified'):
        the_diff = difflib.unified_diff(from_lines, to_lines, args[0], args[1],
                                        from_time, to_time, n=context_lines)
    elif diff_type in ('c', 'context'):
        the_diff = difflib.context_diff(from_lines, to_lines, args[0], args[1],
                                        from_time, to_time, n=context_lines)
    elif diff_type in ('n', 'ndiff'):
        the_diff = difflib.ndiff(from_lines, to_lines)
    elif diff_type in ('h', 'html'):
        the_diff = difflib.HtmlDiff().make_file(from_lines, to_lines, args[0],
                                                args[1], context=True,
                                                numlines=context_lines)
    else:
        raise CommandException("Diff type '" + options.difftype +
                               "' is unknown", "diff")

    sys.stdout.writelines(the_diff)


def b2n(args):
    binary = "".join(args)

    res = 0
    for b in binary:
        res <<= 1
        if b == '1':
            res |= 1
        elif b == '0':
            pass
        else:
            print >> sys.stderr, '"%s" is not a binary number' % binary
            exit(9999)
    print "Bin: %s (%d bits), dec: %d, hex: %x (0x%x)" % \
          (binary, len(binary), res, res, res)


def _num_to_bin(bnum):
    bits = []
    while bnum > 0:
        if bnum & 1:
            bits.insert(0, '1')
        else:
            bits.insert(0, '0')
        bnum >>= 1
    return "".join(bits)


def h2n(args):
    result = 0
    for a in args:
        try:
            num = int(a, 16)
        except ValueError:
            print >> sys.stderr, '"%s" is not a hex number' % a
            result = 9999
            continue

        binary = _num_to_bin(num)
        print "Bin: %s (%d bits), dec: %d, hex: %x (0x%x)" % \
              (binary, len(binary), num, num, num)

    return result


def d2n(args):
    result = 0
    for a in args:
        try:
            num = int(a, 10)
        except ValueError:
            print >> sys.stderr, '"%s" is not a decimal number' % a
            result = 9999
            continue

        binary = _num_to_bin(num)
        print "Bin: %s (%d bits), dec: %d, hex: %x (0x%x)" % \
              (binary, len(binary), num, num, num)

    return result


def load_options():
    types_default = "bat, cs, cpp, c, cc, cxx, h, hhh, hpp, hxx, h++, hs, " \
                    "ii, ixx, ipp, java, js, sh, tpp, txx, tpl, pl, py, rb, " \
                    "xml, shader, xaml"
    usage = "usage: %s options" % sys.argv[0]
    parser = OptionParser(usage)
    parser.add_option("-p", "--path", dest="path",
                      help="The root path", default='.')
    parser.add_option("-r", "--regex", dest="regex", action="append",
                      help="The regex search expression (use quotation "
                           "marks).", default=None)
    parser.add_option("-e", "--end", dest="end",
                      help='End regex. If this is used, the first regex '
                           'becomes the start of a block and the end is '
                           'the end.',
                      default=None)
    parser.add_option("-n", "--numbers", dest="numbers", action="store_true",
                      help="Print line numbers.")
    parser.add_option("-B", "--before", dest="before", type="int",
                      help="Print lines before hit.")
    parser.add_option("-A", "--after", dest="after", type="int",
                      help="Print lines after hit.")
    parser.add_option("-f", "--files", dest="files", action="store_true",
                      help="Just print the list of files.")
    parser.add_option("-i", "--ignorecase", dest="ignorecase",
                      action="store_true",
                      help="Ignore case.")
    parser.add_option("-t", "--types", dest="types", default=types_default,
                      help="File types. Comma separated file endings. "
                           "Default: " + types_default + ".")
    parser.add_option("-s", "--shunpaths", dest="shunpaths", action="append",
                      default=[],
                      help="Shun paths that match patterns defined by -s.")
    parser.add_option("-x", "--shunnames", dest="shunnames", action="append",
                      default=[],
                      help="Shun names that match patterns defined by -x.")
    parser.add_option("-y", "--shunroots", dest="shunroots", action="append",
                      default=[],
                      help="Shun roots that match patterns defined by -y.")
    parser.add_option("-d", "--dirs", dest="dirs", action="store_true",
                      help="Include directories in file name search. Has no "
                           "effect on string match searches (is ignored in "
                           "reader).")
    parser.add_option("--tadd", dest="tadd",
                      help="Add file types. Comma separates file endings.")
    parser.add_option("--tsub", dest="tsub",
                      help="Remove file types. Comma separates file endings.")
    parser.add_option("-b", "--bottomup", dest="bottomup", action="store_true",
                      default=False,
                      help="Traverse bottom up instead of default top down.")
    parser.add_option("-c", "--cute", dest="cute", action="store_true",
                      help="Cute printing.")
    parser.add_option("--detail", dest="detail", action="store_true",
                      help="Detailed printing.")
    parser.add_option("--exe", dest="exe",
                      help='Execute python code for each hit. '
                           'In-parameters are root, file_name, dir_name, '
                           'file_hash, index, line, joined, file_size. They '
                           'may be None.')
    parser.add_option("--remove", dest="remove", action="store_true",
                      help="Print file with hit lines removed or remove when "
                           "running file write commands.")
    parser.add_option("--newfile", dest="newfile", action="store_true",
                      help='Perform operations to a new file where '
                           'applicable (such as removing lines). This is '
                           'selected before "inplace" if both are provided.')
    parser.add_option("--inplace", dest="inplace", action="store_true",
                      help="Perform operations inplace where applicable "
                           "(such as removing lines).")
    parser.add_option("--replacee", dest="replacee",
                      help="Replace all occurrences matched with replacee "
                           'with the tex in "--replacer". If replacee is not '
                           'set, the "--regex" will be used.')
    parser.add_option("--replacer", dest="replacer",
                      help="Replace all occurrences matched with replacee "
                           'with the text in "--replacer". If replacee is '
                           'not set, the "--regex" will be used. "--regex" '
                           'always selects the line.')
    parser.add_option("--version", dest="version", action="store_true",
                      help="Print version (%s)." % VERSION)
    parser.add_option("--license", dest="license", action="store_true",
                      help="Print license infos.")
    parser.add_option("--fnd", dest="fnd", action="store_true",
                      help="Find a file name.")
    parser.add_option("-a", "--all", dest="all", action="store_true",
                      help="No file ending filter (ignores --types).")
    parser.add_option("--wc", dest="wc", action="store_true",
                      help="Word count.")
    parser.add_option("--hints", dest="hints", action="store_true",
                      help="Some usage hints.")
    parser.add_option("--sort", dest="sort", action="store_true",
                      help="Insert buffered sort filter.")
    parser.add_option("--reverse", dest="reverse", action="store_true",
                      default=False, help="Reverse sort filter.")
    parser.add_option("--size", dest="size",
                      help="Base size filter on file size.")
    parser.add_option("--below", dest="below", action="store_true",
                      default=False,
                      help="If used with --size, pass files below size limit.")
    parser.add_option("--md5", dest="md5", action="store_true",
                      help="Calculate md5 sum for file.")
    parser.add_option("--sha1", dest="sha1", action="store_true",
                      help="Calculate sha1 sum for file.")
    parser.add_option("--dups", dest="dups", action="store_true",
                      help="Find duplicate files (by hash).")
    parser.add_option("--zip", dest="zip",
                      help="Zip the found files to zip with the provided "
                           "name.")
    parser.add_option("--time", dest="time", action="store_true",
                      help="Add time stamp to zip file.")
    parser.add_option("--filterinfo", dest="filterinfo", action="store_true",
                      help="Print filter info.")
    parser.add_option("--pipeinfo", dest="pipeinfo", action="store_true",
                      help="Print pipe info and exit.")
    parser.add_option("--build", dest="build", action="store_true",
                      help="Build a pipe on the command line.")
    parser.add_option("--dirsonly", dest="dirsonly", action="store_true",
                      help="Let only directories through.")
    parser.add_option("--rstrip", dest="rstrip", action="store_true",
                      default=False, help='Remove trailing spaces.')
    parser.add_option("--crlf", dest="crlf", action="store_true",
                      default=False, help='Use CRLF line endings. Default is '
                                          'just LF. Only applied together '
                                          'with --rstrip')
    parser.add_option("--diff", dest="diff", action="store_true",
                      default=False, help='Produce a unified diff. Use '
                                          '--difftype for other formats ')
    parser.add_option("--difftype", dest="difftype", default='u',
                      help='Diff type "u" for "unified" (default), "c" for '
                           '"context", "h" for "html" or "n" for "ndiff"')
    parser.add_option("--difflines", dest="difflines", default='3',
                      help='The number of diff context lines. Default 3.')
    parser.add_option("--b2n", dest="b2n", action="store_true",
                      help='Print decimal and hex representation for binary '
                           'number.')
    parser.add_option("--h2n", dest="h2n", action="store_true",
                      help='Print decimal and binary representation for hex '
                           'number.')
    parser.add_option("--d2n", dest="d2n", action="store_true",
                      help='Print hex and binary representation for decimal '
                           'number.')
    parser.add_option("--nocolors", dest="nocolors", action="store_true",
                      help='No colored output.')

    (opts, arguments) = parser.parse_args()
    return opts, arguments


def validate_options():
    options.colors = not IS_DOS and not options.nocolors

    if options.filterinfo:
        filter_info()
        exit(0)

    if options.version:
        print "Version:", VERSION
        exit(0)

    if options.license:
        print LICENSE
        exit(0)

    if options.hints:
        print HINT
        exit(0)

    if options.diff:
        diff(the_args)
        exit(0)

    if options.b2n:
        b2n(the_args)
        exit(0)

    if options.h2n:
        exit(h2n(the_args))

    if options.d2n:
        exit(d2n(the_args))

    if options.dirsonly:
        options.dirs = True

    if options.build:
        if not the_args:
            raise PipeException("--build needs arguments", 24)
        options.build = the_args
        return options

    if not options.regex:
        options.regex = []

    if the_args:
        options.regex += the_args

    if len(options.regex) == 0:
        if options.fnd:
            options.regex.append("")
        else:
            raise PipeException("Malformed search regex (perhaps use "
                                "quotation marks?)", 8)

    if options.files and options.regex[0] in ("", None):
        options.regex = [""]

    if options.regex is None:
        raise PipeException("Search regex empty", 10)

    if options.dups and not options.fnd:
        raise PipeException('"--dups" only works with "--fnd"', 20)

    if options.tadd:
        adds = set([e.strip() for e in options.tadd.split(',')])
        types = set([e.strip() for e in options.types.split(',')])
        options.types = ",".join(list(adds.union(types)))

    if options.tsub:
        subs = set([e.strip() for e in options.tsub.split(',')])
        types = set([e.strip() for e in options.types.split(',')])
        options.types = ",".join(list(types.difference(subs)))

    return options


def main():
    global options, the_args, pipe, pipeException, commandException
    try:
        options, the_args = load_options()
        options = validate_options()
        pipe = build_pipe()
        pipe.process()
    except PipeException, pipeException:
        print "Error: %s, exit code: %d" % (pipeException.message,
                                            pipeException.error_code)
        exit(pipeException.error_code)
    except CommandException, commandException:
        print 'Error: command "%s" failed with message: "%s"' % \
              (commandException.command, commandException.message)
        exit(9999)
    except KeyboardInterrupt:
        print ""
        print "OK, bye..."


if __name__ == '__main__':
    main()
